# -*- coding: utf-8 -*-
import scrapy
import json
import time
from scrapy.http import Request
from lxml import etree
from music.items import CommentItem


class WangyiSpider(scrapy.Spider):
    name = 'wangyi'
    allowed_domains = ['music.163.com']
    # 歌名列表
    songs_name = []
    # 歌曲id列表
    songs_id = []
    # 歌手信息API
    singer_url = 'https://music.163.com/artist?id=893259'
    # 歌曲评论信息API
    song_url = 'http://music.163.com/api/v1/resource/comments/R_SO_4_%s?limit=20&offset=%s'

    def start_requests(self):
        # 获取歌手下的所有歌曲信息，构造歌曲请求url
        self.logger.debug('爬取所有歌曲')
        yield Request(self.singer_url, callback=self.parse_comment)

    def parse_song(self, response):
        # 构造歌曲请求url，得到所有歌曲的id和name信息
        text = etree.HTML(response.text)
        self.songs_name = text.xpath('//div[@id="hotsong-list"]/div[@class="f-cb"]/div/ul//a/text()')
        self.songs_id = text.xpath('//div[@id="hotsong-list"]/div[@class="f-cb"]/div/ul//a/@href') # 获取歌曲id
        self.songs_id = [s_id[9:] for s_id in self.songs_id]
        print('歌曲名列表：' + str(self.songs_name))
        print('歌曲id列表' + str(self.songs_id))
        for song_id in self.songs_id:
            comment_first = self.song_url % (song_id, str(0))
            yield Request(comment_first, callback=self.parse_comment)

    def parse_comment(self, response):
        text = etree.HTML(response.text)
        self.songs_name = text.xpath('//div[@id="hotsong-list"]/div[@class="f-cb"]/div/ul//a/text()')
        self.songs_id = text.xpath('//div[@id="hotsong-list"]/div[@class="f-cb"]/div/ul//a/@href') # 获取歌曲id
        self.songs_id = [s_id[9:] for s_id in self.songs_id]
        print('歌曲名列表：' + str(self.songs_name))
        print('歌曲id列表' + str(self.songs_id))
        for song_id in self.songs_id:
            # 每首歌固定爬取600页数据
            for i in range(600):
                self.logger.debug('开始爬取 ' + song_id + ' 的信息')
                comments_url = self.song_url % (song_id, str(i * 20))
                yield Request(comments_url, callback=self.parse)

    # 将json数据进行转换
    def __json2dict(self, json_text):
        return json.loads(json_text)

    def parse(self, response):
        item = CommentItem()
        json_dict = self._WangyiSpider__json2dict(response.text)
        self.logger.debug('返回评论内容为: ' + str(json_dict))
        for comment in json_dict['comments']:
            item['comment'] = comment['content'].replace('\n', '')
            item['liked_count'] = comment['likedCount']
            item['comment_time'] = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(comment['time'] / 1000))
            yield item
